package com.g2room.myuall.grap.ky;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.g2room.driving.grap.GenericGrap;
import com.g2room.myuall.grap.ky.model.po.DicType;
import com.g2room.myuall.grap.ky.model.po.GrapErr;
import com.g2room.myuall.grap.ky.model.po.PeeArti;
import com.g2room.myuall.grap.ky.model.po.PeeUnivInfo;
import com.g2room.myuall.util.DateUtil;
import com.g2room.myuall.util.DbUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

/**
 * 根据考研大学抓取相关信息
 * @author butterfly
 *
 */
public class GrapKyArti {
//	考研kaoyan.com
//	留学http://www.liuxue86.com/
//	论文http://biyelunwen.yjbys.com/
	// 抓取新闻== http://www.kaoyan.com/xinwen/
	// http://yz.kaoyan.com/ 985 211 34所
	private static Map<String, Integer> infosMap = new HashMap<String, Integer>();
	private static Map<Integer, DicType> types=getType();
	static {
		// 报考指南
		/**
		 * 24	报考指南
25	报考经验
26	择校择专业
		 */
		infosMap.put("http://www.kaoyan.com/baokao/zhinan/",24);
		infosMap.put("http://www.kaoyan.com/baokao/jingyan/",25);
		infosMap.put("http://www.kaoyan.com/baokao/zexiao/",26);
		// 新闻资讯
//		27	政策新闻
//		28	时评看点
//		30	求职就业
		infosMap.put("http://www.kaoyan.com/xinwen/zhengce/",27);
		infosMap.put("http://www.kaoyan.com/xinwen/kandian/",28);
		infosMap.put("http://www.kaoyan.com/xinwen/jiuye/",30);
//		18	复习经验
//		29	每日一练
//		33	考研大纲解析
//		34	考研真题解析
		// 专业课
		infosMap.put("http://www.kaoyan.com/zhuanyeke/lianxi/",29);
		infosMap.put("http://www.kaoyan.com/zhuanyeke/jingyan/",18);
		infosMap.put("http://www.kaoyan.com/zhuanyeke/zhenti/",34);
		infosMap.put("http://www.kaoyan.com/zhuanyeke/dagang/",33);
		
//		17	备考经验
//		22	心理调节
//		23	心路历程
//		32	考研复习
		// 备考宝典
		infosMap.put("http://www.kaoyan.com/beikao/fuxi/",32);
		infosMap.put("http://www.kaoyan.com/beikao/jingyan/",17);
		infosMap.put("http://www.kaoyan.com/beikao/xinlu/",23);
		infosMap.put("http://www.kaoyan.com/beikao/xinli/",22);
		
//		31	经验技巧
//		35	英语作文
//		36	英语口语
//		37	英语听力
//		38	英语朗读
//		39	英语翻译
//		40	英语考试
//		41	英语词汇
//		42	英语语法
//		43	英语阅读
		// 英语学习
        infosMap.put("http://www.kaoyan.com/english/grammar/",42);
        infosMap.put("http://www.kaoyan.com/english/reading/",43);
        infosMap.put("http://www.kaoyan.com/english/translate/",39);
        infosMap.put("http://www.kaoyan.com/english/writing/",35);
        infosMap.put("http://www.kaoyan.com/english/exam/",40);
        infosMap.put("http://www.kaoyan.com/english/skills/",31);
        infosMap.put("http://www.kaoyan.com/english/speaking/",36);
        infosMap.put("http://www.kaoyan.com/english/words/",41);
        infosMap.put("http://www.kaoyan.com/english/englishreading/",38);
        infosMap.put("http://www.kaoyan.com/english/listening/",37);
        
//        48	每日一练
//        49	复习经验
//        50	视频课程
//        51	考研真题解析
//        52	考研大纲解析
        // 数学
        infosMap.put("http://www.kaoyan.com/shuxue/lianxi/",48);
        infosMap.put("http://www.kaoyan.com/shuxue/jingyan/",49);
        infosMap.put("http://www.kaoyan.com/shuxue/shipin/",50);
        infosMap.put("http://www.kaoyan.com/shuxue/zhenti/",51);
        infosMap.put("http://www.kaoyan.com/shuxue/dagang/",52);
        
//        53	每日一练
//        54	复习经验
//        55	视频课程
//        56	考研真题解析
//        57	考研大纲解析
        // 政治
        infosMap.put("http://www.kaoyan.com/zhengzhi/lianxi/",53);
        infosMap.put("http://www.kaoyan.com/zhengzhi/jingyan/",54);
        infosMap.put("http://www.kaoyan.com/zhengzhi/shipin/",55);
        infosMap.put("http://www.kaoyan.com/zhengzhi/zhenti/",56);
        infosMap.put("http://www.kaoyan.com/zhengzhi/dagang/",57);
        
//        11	MBA备考
//        12	MBA复试
//        13	MBA报考
//        15	MBA资讯
//        16	人物案例
        //MBA
        infosMap.put("http://mba.kaoyan.com/zixun/",15);
        infosMap.put("http://mba.kaoyan.com/renwu/",16);
        infosMap.put("http://mba.kaoyan.com/baokao/",13);
        infosMap.put("http://mba.kaoyan.com/beikao/",11);
        infosMap.put("http://mba.kaoyan.com/fushi/",12);
        infosMap.put("http://mba.kaoyan.com/tiaoji/",14);
//        19	复试信息
//        20	复试新闻
//        21	复试经验
        // 考研复试);
        infosMap.put("http://www.kaoyan.com/fushi/xinxi/",19);
        infosMap.put("http://www.kaoyan.com/fushi/xinwen/",20);
        infosMap.put("http://www.kaoyan.com/fushi/jingyan/",21);
        
//        45	调剂信息
//        46	调剂新闻
//        47	调剂经验
        infosMap.put("http://tiaoji.kaoyan.com/xinxi/",45);
        infosMap.put("http://tiaoji.kaoyan.com/xinwen/",46);
        infosMap.put("http://tiaoji.kaoyan.com/jingyan/",47);
	}
	private String url;
	private Integer type;
	public GrapKyArti(String url, Integer type) {
		this.url = url;
		this.type = type;
	}
	public static void main(String[] args) throws Exception{
		for(String url : infosMap.keySet()) {
			GrapKyArti gka = new GrapKyArti(url, infosMap.get(url));
			gka.getPage();
		}
	}
	
	private void getPage() {
		System.out.println(type + "===" + url);
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (Exception e) {
			System.out.println(url);
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("10");
			ge.setContent(JSON.toJSONString(types.get(type)));
			ge.setDescription("获取News  Page错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		
		if(doc == null) return ;
		Elements pageEls = doc.getElementsByAttributeValue("class", "tPage");
		List<String> urls = new ArrayList<String>();
		urls.add(url);
		if(null != pageEls && !pageEls.isEmpty()) {
			// 获取页码
			Elements aEls = pageEls.first().getElementsByTag("a");
			if(aEls != null && aEls.size() > 2) {
				Integer pageNum = Integer.valueOf(aEls.get(aEls.size() -2).text());
				for(int i = 2; i<= pageNum; i ++) {
					urls.add(url + "index_"+i+".html");
				}
			}
		}
		for(String listUrl : urls) {
			getList(listUrl);
		}
	}
	
	/**
	 * 获取分页列表
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getList(String listUrl) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(listUrl);
			if(doc == null) return;
			
	//		doc.getElementsByAttributeValueContaining(key, match)
			Elements aEls = doc.getElementsByAttributeValueContaining("class", "areaZslist").first().getElementsByTag("a");
			List<String> urls = new ArrayList<String>();
			for(Element aEl : aEls) {
				urls.add(aEl.attr("href"));
			}

	        for(String detailUrl : urls) {
	        	getDetail(detailUrl);
	        }
		} catch (Exception e) {
			System.out.println(listUrl);
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(listUrl);
			ge.setType("12");
			ge.setContent(JSON.toJSONString(types.get(type)));
			ge.setDescription("获取News  List错误："+e.getMessage());
			insert2ErrDb(ge);
		}
	}

	/**
	 * 获取List列表
	 * @param url
	 */
	public void getDetail(String detailUrl) {
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(detailUrl);
			if(doc == null) return;
			
			PeeArti pa = new PeeArti();
			pa.setSourceUrl(detailUrl);
			DicType dt = types.get(type);
			pa.setTypeId(dt.getId());
			pa.setTypeName(dt.getName());
			pa.setCategoryId(dt.getParentId());
			pa.setCategoryName(types.get(dt.getParentId()).getName());
			
			Elements els = doc.getElementsByAttributeValue("class", "articleTitle");
			if(null != els && !els.isEmpty()) {
				pa.setTitle(els.first().text());
			} else {
				return;
			}
			els = doc.getElementsByAttributeValue("class", "articleCon");
			if(null != els && !els.isEmpty()) {
				pa.setContent(els.first().html());
			} else {
				return;
			}
			els = doc.getElementsByAttributeValue("class", "articleInfo");
			if(null != els && !els.isEmpty()) {
				Elements spanEls = els.first().getElementsByTag("span");
				if(spanEls.size() > 0) {
					// 2004-09-08 16:22
					String publishTime = spanEls.get(0).text();
					try {
						pa.setPublishTime(DateUtil.string2Date(publishTime, "yyyy-MM-dd HH:mm"));
					} catch (ParseException e) {
						e.printStackTrace();
					}
					if(null != spanEls.get(1)) {
						pa.setSource(spanEls.get(1).text());
					}
				}
			}
			insert2NewsDb(pa);
		} catch (Exception e) {
			System.out.println(url);
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("13");
			ge.setContent(e.getMessage());
			ge.setDescription("获取Detail错误");
			insert2ErrDb(ge);
		}
	}
	

	public static void insert2NewsDb(PeeArti pg) throws Exception {
		String sql = "insert into pee_arti(type_name,content,source_url,source,title,publish_time,type_id,category_name,category_id) values(?,?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = DbUtil.getConn().prepareStatement(sql);
			// type_name,content,source_url,source,title,publish_time,type_id,category_name,category_id
			ps.setString(1, pg.getTypeName());
			ps.setString(2, pg.getContent());
			ps.setString(3, pg.getSourceUrl());
			ps.setString(4, pg.getSource());
			ps.setString(5, pg.getTitle());
			ps.setDate(6, new java.sql.Date(pg.getPublishTime().getTime()));
			ps.setInt(7, pg.getTypeId());
			ps.setString(8, pg.getCategoryName());
			ps.setInt(9, pg.getCategoryId());
			ps.execute();	
		} catch (Exception e) {
			e.printStackTrace();
			throw e;
		} finally {
			if(null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}
	
	public void insert2ErrDb(GrapErr ge) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getConn().prepareStatement("insert into grap_err(description,url,content,type) values(?,?,?,?)");
			grapPs.setString(1, ge.getDescription());
			grapPs.setString(2, ge.getUrl());
			grapPs.setString(3, ge.getContent());
			grapPs.setString(4, ge.getType());
			grapPs.execute();
			grapPs.close();
		} catch(Exception e) {
			e.printStackTrace();
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}
	

	public static Map<Integer, DicType> getType() {
		Map<Integer, DicType> univs = new HashMap<Integer, DicType>();
		try {
			Connection connection = DbUtil.getConn();
			Statement statement = connection.createStatement();
			String sql = "select * from dic_type";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				DicType dt = new DicType();
				dt.setId(rs.getInt("id"));
				dt.setName(rs.getString("name"));
				dt.setParentId(rs.getInt("parent_id"));
				
				univs.put(dt.getId(), dt);
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return univs;
	}
}
